survival plot session VS spark node

load libraries

library(survival)
library(survminer)
library(cgdsr)
library(sparklyr)
library(dplyr)

R Session Processing

Load clinical Data

cgds <- cgdsr::CGDS("http://www.cbioportal.org/public-portal/")
#Studies<- cgdsr::getCancerStudies(cgds)
clinicalData <- cgdsr::getClinicalData(cgds, "gbm_tcga_pub_all")
#clinicalData <- read.csv("Clinical_tab.csv") #, na.strings=c("","NA")

Transformations 1

clinicalData$OS_STATUS <- gsub("LIVING", "0", clinicalData$OS_STATUS, ignore.case = TRUE)
clinicalData$OS_STATUS <- gsub("DECEASED", "1", clinicalData$OS_STATUS, ignore.case = TRUE)
clinicalData$DFS_STATUS <- gsub("^$|^ $", "DiseaseFree", clinicalData$DFS_STATUS, ignore.case = TRUE)
clinicalData$OS_STATUS <- as.numeric(clinicalData$OS_STATUS)

survival plot

fit <- survival::survfit(Surv(OS_MONTHS, OS_STATUS) ~ DFS_STATUS, data = clinicalData)
   survminer::ggsurvplot(fit, data = clinicalData,
                          type = "kaplan-meier",
                          #conf.type="log",
                          conf.int = TRUE,
                          pval = TRUE,
                          fun = "pct",
                          risk.table = TRUE,
                          size = 1,
                          linetype = "strata",
                          palette = c("#E7B800", "#2E9FDF"),
                          legend = "top",
                          lengend.title = "DFS_STATUS",
                          legend.labs = c("DiseaseFree", "Recurred")
   )

R Session: Plot DiseaseFree vs Reccured during OS_MONTHS

  clinicalData <- cgdsr::getClinicalData(cgds, "gbm_tcga_pub_all")
start_time <- Sys.time()
  clinicalData %>% 
  mutate(OS_STATUS = gsub("LIVING", "0", OS_STATUS)) %>%
  mutate(OS_STATUS = gsub( "DECEASED", "1", OS_STATUS)) %>%
  mutate(DFS_STATUS = gsub( "^$|^ $", "DiseaseFree", DFS_STATUS)) %>%
  mutate(OS_STATUS = as.numeric(OS_STATUS)) %>%
  arrange(OS_MONTHS) %>%
  mutate( DiseaseFree = ifelse(DFS_STATUS == "DiseaseFree", 1, 0)) %>% 
  as.data.frame() %>%
  mutate(n_DiseaseFree = cumsum(DiseaseFree == 1)) %>%
  mutate(n_Recurred = cumsum(DiseaseFree == 0)) %>%
  ggplot(aes(x = OS_MONTHS, y = value, color = variable)) +
  geom_point(aes(y = n_DiseaseFree, col = "n_DiseaseFree")) +
  geom_point(aes(y = n_Recurred, col = "n_Recurred")) +
  labs(title = paste("Using R Session, Running time = ", Sys.time() - start_time))

Spark Node: Plot DiseaseFree vs Reccured during OS_MONTHS

 clinicalData <- cgdsr::getClinicalData(cgds, "gbm_tcga_pub_all")
 sc <- spark_connect(master = "local",
                     version = "2.4.0")

Re-using existing Spark connection to local

 clinicalData_tbl <- dplyr::copy_to(sc, clinicalData, overwrite = TRUE)
  start_time <- Sys.time()
  clinicalData_tbl %>%
  mutate(OS_STATUS = regexp_replace(OS_STATUS, "LIVING", "0")) %>%
  mutate(OS_STATUS = regexp_replace(OS_STATUS, "DECEASED", "1")) %>%
  mutate(DFS_STATUS = regexp_replace(DFS_STATUS, "^$|^ $", "DiseaseFree")) %>%
  mutate(OS_STATUS = as.numeric(OS_STATUS)) %>%
  #mutate(OS_STATUS = regexp_replace(as.numeric(OS_STATUS), 'NaN', NA)) %>%
  #mutate(OS_STATUS = regexp_replace(OS_STATUS, NaN, NA)) %>%
  #na.replace('') %>%  ## not good for OS_STATUS (0,1)
  #dplyr::filter(!is.na(OS_MONTHS)) 
  arrange(is.na(OS_MONTHS), OS_MONTHS) %>%  ## OUFFF put Nan at the end of the column
  mutate(DiseaseFree = ifelse(DFS_STATUS == "DiseaseFree", 1, 0)) %>% 
  as.data.frame() %>%
  mutate( n_DiseaseFree = cumsum(as.numeric(DiseaseFree == 1 ))) %>%
  mutate( n_Recurred = cumsum(as.numeric(DiseaseFree == 0 ))) %>%
  ggplot(aes(x = OS_MONTHS, y = value, color = variable)) +
  geom_point(aes(y = n_DiseaseFree, col = "n_DiseaseFree")) +
  geom_point(aes(y = n_Recurred, col = "n_Recurred"))  +
   labs(title = paste("Using Spark Node, Running time = ", Sys.time() - start_time))